<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta charset="utf-8" />
    <title>geosnap.io.storage &#8212; geosnap v0.3.1 Manual</title>
    <link rel="stylesheet" href="../../../_static/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" type="text/css" href="../../../_static/pysal-styles.css" />
    <script id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
    <script src="../../../_static/jquery.js"></script>
    <script src="../../../_static/underscore.js"></script>
    <script src="../../../_static/doctools.js"></script>
    <script src="../../../_static/language_data.js"></script>
    <link rel="shortcut icon" href="../../../_static/geosnap.ico"/>
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" />
<meta charset='utf-8'>
<meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
<meta name="apple-mobile-web-app-capable" content="yes">
<script type="text/javascript" src="../../../_static/js/jquery-1.11.0.min.js "></script>
<script type="text/javascript" src="../../../_static/js/jquery-fix.js "></script>
<script type="text/javascript" src="../../../_static/bootstrap-3.3.7/js/bootstrap.min.js "></script>
<script type="text/javascript" src="../../../_static/bootstrap-sphinx.js "></script>

  </head><body>

  <div id="navbar" class="navbar navbar-default navbar-fixed-top">
    <div class="container">
      <div class="navbar-header">
        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
          <span class="icon-bar"></span>
          <span class="icon-bar"></span>
          <span class="icon-bar"></span>
        </button>
        <a class="navbar-brand" href="../../../index.html"><span><img src="../../../_static/geosnap.svg"></span>
          geosnap</a>
        <span class="navbar-text navbar-version pull-left"><b>0.3.1</b></span>
      </div>

        <div class="collapse navbar-collapse nav-collapse">
          <ul class="nav navbar-nav">
            
                <li><a href="../../../installation.html">Installation</a></li>
                <li><a href="../../../api.html">API</a></li>
                <li><a href="../../../data.html">Data</a></li>
                <li><a href="https://spatialucr.github.io/geosnap-guide/">User Guide</a></li>
                <li><a href="../../../references.html">References</a></li>
            
            
              <li class="dropdown globaltoc-container">
  <a role="button"
     id="dLabelGlobalToc"
     data-toggle="dropdown"
     data-target="#"
     href="../../../index.html">Site <b class="caret"></b></a>
  <ul class="dropdown-menu globaltoc"
      role="menu"
      aria-labelledby="dLabelGlobalToc"><p class="caption"><span class="caption-text">Contents:</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../installation.html#installing-a-released-version">Installing a released version</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../installation.html#installing-development-version">Installing development version</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">API</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../api.html#io-module">IO Module</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../api.html#the-community-class">The Community Class</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../api.html#analyze-module">Analyze Module</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../api.html#harmonize-module">Harmonize Module</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../api.html#visualize-module">Visualize Module</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../references.html">References</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../data.html">Data</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../data.html#tabular-data">Tabular Data</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../data.html#geo-data">Geo Data</a></li>
</ul>
</li>
</ul>
</ul>
</li>
              
                <li class="dropdown">
  <a role="button"
     id="dLabelLocalToc"
     data-toggle="dropdown"
     data-target="#"
     href="#">Page <b class="caret"></b></a>
  <ul class="dropdown-menu localtoc"
      role="menu"
      aria-labelledby="dLabelLocalToc"></ul>
</li>
              
            
            
            
            
            
          </ul>

          
            
<form class="navbar-form navbar-right" action="../../../search.html" method="get">
 <div class="form-group">
  <input type="text" name="q" class="form-control" placeholder="Search" />
 </div>
  <input type="hidden" name="check_keywords" value="yes" />
  <input type="hidden" name="area" value="default" />
</form>
          
        </div>
    </div>
  </div>

<div class="container">
  <div class="row">
    <div class="body col-md-12 content" role="main">
      
  <h1>Source code for geosnap.io.storage</h1><div class="highlight"><pre>
<span></span><span class="sd">&quot;&quot;&quot;Tools for creating and manipulating neighborhood datasets.&quot;&quot;&quot;</span>

<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">pathlib</span>
<span class="kn">import</span> <span class="nn">zipfile</span>
<span class="kn">from</span> <span class="nn">warnings</span> <span class="kn">import</span> <span class="n">warn</span>

<span class="kn">from</span> <span class="nn">appdirs</span> <span class="kn">import</span> <span class="n">user_data_dir</span>

<span class="kn">import</span> <span class="nn">geopandas</span> <span class="k">as</span> <span class="nn">gpd</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="kn">import</span> <span class="nn">quilt3</span>

<span class="kn">from</span> <span class="nn">.._data</span> <span class="kn">import</span> <span class="n">datasets</span>
<span class="kn">from</span> <span class="nn">.util</span> <span class="kn">import</span> <span class="n">adjust_inflation</span><span class="p">,</span> <span class="n">convert_gdf</span>

<span class="n">_fipstable</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span>
    <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)),</span> <span class="s2">&quot;stfipstable.csv&quot;</span><span class="p">),</span>
    <span class="n">converters</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;FIPS Code&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">},</span>
<span class="p">)</span>

<span class="n">appname</span> <span class="o">=</span> <span class="s2">&quot;geosnap&quot;</span>
<span class="n">appauthor</span> <span class="o">=</span> <span class="s2">&quot;geosnap&quot;</span>
<span class="n">data_dir</span> <span class="o">=</span> <span class="n">user_data_dir</span><span class="p">(</span><span class="n">appname</span><span class="p">,</span> <span class="n">appauthor</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">data_dir</span><span class="p">):</span>
    <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="n">data_dir</span><span class="p">)</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="c1"># look for local storage and create if missing</span>
<span class="k">try</span><span class="p">:</span>
    <span class="n">storage</span> <span class="o">=</span> <span class="n">quilt3</span><span class="o">.</span><span class="n">Package</span><span class="o">.</span><span class="n">browse</span><span class="p">(</span><span class="s2">&quot;geosnap_data/storage&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
    <span class="n">storage</span> <span class="o">=</span> <span class="n">quilt3</span><span class="o">.</span><span class="n">Package</span><span class="p">()</span>


<div class="viewcode-block" id="store_census"><a class="viewcode-back" href="../../../generated/geosnap.io.store_census.html#geosnap.io.store_census">[docs]</a><span class="k">def</span> <span class="nf">store_census</span><span class="p">(</span><span class="n">dest</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Save census data to the local quilt package storage.</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    None</span>
<span class="sd">        Data will be available in the geosnap.data.datasets and will be used</span>
<span class="sd">        in place of streaming data for all census queries. The raster package</span>
<span class="sd">        is 3.05 GB.</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">quilt3</span><span class="o">.</span><span class="n">Package</span><span class="o">.</span><span class="n">install</span><span class="p">(</span><span class="s2">&quot;census/tracts_cartographic&quot;</span><span class="p">,</span> <span class="s2">&quot;s3://spatial-ucr&quot;</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="n">dest</span><span class="p">)</span>
    <span class="n">quilt3</span><span class="o">.</span><span class="n">Package</span><span class="o">.</span><span class="n">install</span><span class="p">(</span><span class="s2">&quot;census/administrative&quot;</span><span class="p">,</span> <span class="s2">&quot;s3://spatial-ucr&quot;</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="n">dest</span><span class="p">)</span></div>


<div class="viewcode-block" id="store_blocks_2000"><a class="viewcode-back" href="../../../generated/geosnap.io.store_blocks_2000.html#geosnap.io.store_blocks_2000">[docs]</a><span class="k">def</span> <span class="nf">store_blocks_2000</span><span class="p">(</span><span class="n">dest</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Save census 2000 census block data to the local quilt package storage.</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    None</span>
<span class="sd">        Data will be available in the geosnap.data.datasets and will be used</span>
<span class="sd">        in place of streaming data for all census queries.</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">quilt3</span><span class="o">.</span><span class="n">Package</span><span class="o">.</span><span class="n">install</span><span class="p">(</span><span class="s2">&quot;census/blocks_2000&quot;</span><span class="p">,</span> <span class="s2">&quot;s3://spatial-ucr&quot;</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="n">dest</span><span class="p">)</span></div>


<div class="viewcode-block" id="store_blocks_2010"><a class="viewcode-back" href="../../../generated/geosnap.io.store_blocks_2010.html#geosnap.io.store_blocks_2010">[docs]</a><span class="k">def</span> <span class="nf">store_blocks_2010</span><span class="p">(</span><span class="n">dest</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Save census 2010 census block data to the local quilt package storage.</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    None</span>
<span class="sd">        Data will be available in the geosnap.data.datasets and will be used</span>
<span class="sd">        in place of streaming data for all census queries.</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">quilt3</span><span class="o">.</span><span class="n">Package</span><span class="o">.</span><span class="n">install</span><span class="p">(</span><span class="s2">&quot;census/blocks_2010&quot;</span><span class="p">,</span> <span class="s2">&quot;s3://spatial-ucr&quot;</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="n">dest</span><span class="p">)</span></div>


<div class="viewcode-block" id="store_ltdb"><a class="viewcode-back" href="../../../generated/geosnap.io.store_ltdb.html#geosnap.io.store_ltdb">[docs]</a><span class="k">def</span> <span class="nf">store_ltdb</span><span class="p">(</span><span class="n">sample</span><span class="p">,</span> <span class="n">fullcount</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Read &amp; store data from Brown&#39;s Longitudinal Tract Database (LTDB).</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    sample : str</span>
<span class="sd">        file path of the zip file containing the standard Sample CSV files</span>
<span class="sd">        downloaded from</span>
<span class="sd">        https://s4.ad.brown.edu/projects/diversity/Researcher/LTBDDload/Default.aspx</span>

<span class="sd">    fullcount: str</span>
<span class="sd">        file path of the zip file containing the standard Fullcount CSV files</span>
<span class="sd">        downloaded from</span>
<span class="sd">        https://s4.ad.brown.edu/projects/diversity/Researcher/LTBDDload/Default.aspx</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    pandas.DataFrame</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">sample_zip</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
    <span class="n">fullcount_zip</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">fullcount</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_ltdb_reader</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">file</span><span class="p">,</span> <span class="n">year</span><span class="p">,</span> <span class="n">dropcols</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>

        <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span>
            <span class="n">path</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">file</span><span class="p">),</span>
            <span class="n">na_values</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="mi">99999</span><span class="p">,</span> <span class="o">-</span><span class="mi">999</span><span class="p">],</span>
            <span class="n">converters</span><span class="o">=</span><span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="s2">&quot;placefp10&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">},</span>
            <span class="n">low_memory</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
            <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;latin1&quot;</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="k">if</span> <span class="n">dropcols</span><span class="p">:</span>
            <span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">dropcols</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="n">df</span><span class="o">.</span><span class="n">columns</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
        <span class="n">names</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="n">names</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;geoid&quot;</span>
        <span class="n">newlist</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="c1"># ignoring the first 4 columns, remove year suffix from column names</span>
        <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">[</span><span class="mi">4</span><span class="p">:]:</span>
            <span class="n">newlist</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">name</span><span class="p">[:</span><span class="o">-</span><span class="mi">2</span><span class="p">])</span>
        <span class="n">colnames</span> <span class="o">=</span> <span class="n">names</span><span class="p">[:</span><span class="mi">4</span><span class="p">]</span> <span class="o">+</span> <span class="n">newlist</span>
        <span class="n">df</span><span class="o">.</span><span class="n">columns</span> <span class="o">=</span> <span class="n">colnames</span>

        <span class="c1"># prepend a 0 when FIPS is too short</span>
        <span class="n">df</span><span class="p">[</span><span class="s2">&quot;geoid&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s2">&quot;geoid&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">rjust</span><span class="p">(</span><span class="mi">11</span><span class="p">,</span> <span class="s2">&quot;0&quot;</span><span class="p">)</span>
        <span class="n">df</span><span class="o">.</span><span class="n">set_index</span><span class="p">(</span><span class="s2">&quot;geoid&quot;</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

        <span class="n">df</span><span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">year</span>

        <span class="n">inflate_cols</span> <span class="o">=</span> <span class="p">[</span>
            <span class="s2">&quot;mhmval&quot;</span><span class="p">,</span>
            <span class="s2">&quot;mrent&quot;</span><span class="p">,</span>
            <span class="s2">&quot;incpc&quot;</span><span class="p">,</span>
            <span class="s2">&quot;hinc&quot;</span><span class="p">,</span>
            <span class="s2">&quot;hincw&quot;</span><span class="p">,</span>
            <span class="s2">&quot;hincb&quot;</span><span class="p">,</span>
            <span class="s2">&quot;hinch&quot;</span><span class="p">,</span>
            <span class="s2">&quot;hinca&quot;</span><span class="p">,</span>
        <span class="p">]</span>

        <span class="n">inflate_available</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">)</span><span class="o">.</span><span class="n">intersection</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">inflate_cols</span><span class="p">)))</span>

        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">inflate_available</span><span class="p">):</span>
            <span class="n">df</span> <span class="o">=</span> <span class="n">adjust_inflation</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">inflate_available</span><span class="p">,</span> <span class="n">year</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">df</span>

    <span class="c1"># read in Brown&#39;s LTDB data, both the sample and fullcount files for each</span>
    <span class="c1"># year population, housing units &amp; occupied housing units appear in both</span>
    <span class="c1"># &quot;sample&quot; and &quot;fullcount&quot; files-- currently drop sample and keep fullcount</span>

    <span class="n">sample70</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span>
        <span class="n">sample_zip</span><span class="p">,</span>
        <span class="s2">&quot;ltdb_std_all_sample/ltdb_std_1970_sample.csv&quot;</span><span class="p">,</span>
        <span class="n">dropcols</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;POP70SP1&quot;</span><span class="p">,</span> <span class="s2">&quot;HU70SP&quot;</span><span class="p">,</span> <span class="s2">&quot;OHU70SP&quot;</span><span class="p">],</span>
        <span class="n">year</span><span class="o">=</span><span class="mi">1970</span><span class="p">,</span>
    <span class="p">)</span>

    <span class="n">fullcount70</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span><span class="n">fullcount_zip</span><span class="p">,</span> <span class="s2">&quot;LTDB_Std_1970_fullcount.csv&quot;</span><span class="p">,</span> <span class="n">year</span><span class="o">=</span><span class="mi">1970</span><span class="p">)</span>

    <span class="n">sample80</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span>
        <span class="n">sample_zip</span><span class="p">,</span>
        <span class="s2">&quot;ltdb_std_all_sample/ltdb_std_1980_sample.csv&quot;</span><span class="p">,</span>
        <span class="n">dropcols</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;pop80sf3&quot;</span><span class="p">,</span> <span class="s2">&quot;pop80sf4&quot;</span><span class="p">,</span> <span class="s2">&quot;hu80sp&quot;</span><span class="p">,</span> <span class="s2">&quot;ohu80sp&quot;</span><span class="p">],</span>
        <span class="n">year</span><span class="o">=</span><span class="mi">1980</span><span class="p">,</span>
    <span class="p">)</span>

    <span class="n">fullcount80</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span><span class="n">fullcount_zip</span><span class="p">,</span> <span class="s2">&quot;LTDB_Std_1980_fullcount.csv&quot;</span><span class="p">,</span> <span class="n">year</span><span class="o">=</span><span class="mi">1980</span><span class="p">)</span>

    <span class="n">sample90</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span>
        <span class="n">sample_zip</span><span class="p">,</span>
        <span class="s2">&quot;ltdb_std_all_sample/ltdb_std_1990_sample.csv&quot;</span><span class="p">,</span>
        <span class="n">dropcols</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;POP90SF3&quot;</span><span class="p">,</span> <span class="s2">&quot;POP90SF4&quot;</span><span class="p">,</span> <span class="s2">&quot;HU90SP&quot;</span><span class="p">,</span> <span class="s2">&quot;OHU90SP&quot;</span><span class="p">],</span>
        <span class="n">year</span><span class="o">=</span><span class="mi">1990</span><span class="p">,</span>
    <span class="p">)</span>

    <span class="n">fullcount90</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span><span class="n">fullcount_zip</span><span class="p">,</span> <span class="s2">&quot;LTDB_Std_1990_fullcount.csv&quot;</span><span class="p">,</span> <span class="n">year</span><span class="o">=</span><span class="mi">1990</span><span class="p">)</span>

    <span class="n">sample00</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span>
        <span class="n">sample_zip</span><span class="p">,</span>
        <span class="s2">&quot;ltdb_std_all_sample/ltdb_std_2000_sample.csv&quot;</span><span class="p">,</span>
        <span class="n">dropcols</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;POP00SF3&quot;</span><span class="p">,</span> <span class="s2">&quot;HU00SP&quot;</span><span class="p">,</span> <span class="s2">&quot;OHU00SP&quot;</span><span class="p">],</span>
        <span class="n">year</span><span class="o">=</span><span class="mi">2000</span><span class="p">,</span>
    <span class="p">)</span>

    <span class="n">fullcount00</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span><span class="n">fullcount_zip</span><span class="p">,</span> <span class="s2">&quot;LTDB_Std_2000_fullcount.csv&quot;</span><span class="p">,</span> <span class="n">year</span><span class="o">=</span><span class="mi">2000</span><span class="p">)</span>

    <span class="n">sample10</span> <span class="o">=</span> <span class="n">_ltdb_reader</span><span class="p">(</span>
        <span class="n">sample_zip</span><span class="p">,</span> <span class="s2">&quot;ltdb_std_all_sample/ltdb_std_2010_sample.csv&quot;</span><span class="p">,</span> <span class="n">year</span><span class="o">=</span><span class="mi">2010</span>
    <span class="p">)</span>
    <span class="c1"># join the sample and fullcount variables into a single df for the year</span>
    <span class="n">ltdb_1970</span> <span class="o">=</span> <span class="n">sample70</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">])</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
        <span class="n">fullcount70</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span> <span class="mi">7</span><span class="p">:],</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span>
    <span class="p">)</span>
    <span class="n">ltdb_1980</span> <span class="o">=</span> <span class="n">sample80</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">])</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
        <span class="n">fullcount80</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span> <span class="mi">7</span><span class="p">:],</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span>
    <span class="p">)</span>
    <span class="n">ltdb_1990</span> <span class="o">=</span> <span class="n">sample90</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">])</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
        <span class="n">fullcount90</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span> <span class="mi">7</span><span class="p">:],</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span>
    <span class="p">)</span>
    <span class="n">ltdb_2000</span> <span class="o">=</span> <span class="n">sample00</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">])</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
        <span class="n">fullcount00</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span> <span class="mi">7</span><span class="p">:],</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span>
    <span class="p">)</span>
    <span class="n">ltdb_2010</span> <span class="o">=</span> <span class="n">sample10</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">ltdb_1970</span><span class="p">,</span> <span class="n">ltdb_1980</span><span class="p">,</span> <span class="n">ltdb_1990</span><span class="p">,</span> <span class="n">ltdb_2000</span><span class="p">,</span> <span class="n">ltdb_2010</span><span class="p">],</span> <span class="n">sort</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="n">renamer</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
        <span class="nb">zip</span><span class="p">(</span><span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()[</span><span class="s2">&quot;ltdb&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span> <span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()[</span><span class="s2">&quot;variable&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">())</span>
    <span class="p">)</span>

    <span class="n">df</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">renamer</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="s2">&quot;columns&quot;</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="c1"># compute additional variables from lookup table</span>
    <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()[</span><span class="s2">&quot;formula&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">dropna</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span>
        <span class="n">df</span><span class="o">.</span><span class="n">eval</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="n">keeps</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">[</span>
        <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()[</span><span class="s2">&quot;variable&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">])</span>
    <span class="p">]</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">keeps</span><span class="p">]</span>

    <span class="n">df</span><span class="o">.</span><span class="n">to_parquet</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_dir</span><span class="p">,</span> <span class="s2">&quot;ltdb.parquet&quot;</span><span class="p">),</span> <span class="n">compression</span><span class="o">=</span><span class="s2">&quot;brotli&quot;</span><span class="p">)</span>
    <span class="n">storage</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">&quot;ltdb&quot;</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_dir</span><span class="p">,</span> <span class="s2">&quot;ltdb.parquet&quot;</span><span class="p">))</span>
    <span class="n">storage</span><span class="o">.</span><span class="n">build</span><span class="p">(</span><span class="s2">&quot;geosnap_data/storage&quot;</span><span class="p">)</span></div>


<div class="viewcode-block" id="store_ncdb"><a class="viewcode-back" href="../../../generated/geosnap.io.store_ncdb.html#geosnap.io.store_ncdb">[docs]</a><span class="k">def</span> <span class="nf">store_ncdb</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Read &amp; store data from Geolytics&#39;s Neighborhood Change Database.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    filepath : str</span>
<span class="sd">        location of the input CSV file extracted from your Geolytics DVD</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">ncdb_vars</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()[</span><span class="s2">&quot;ncdb&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">dropna</span><span class="p">()[</span><span class="mi">1</span><span class="p">:]</span><span class="o">.</span><span class="n">values</span>

    <span class="n">names</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">ncdb_vars</span><span class="p">:</span>
        <span class="k">for</span> <span class="n">suffix</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;7&quot;</span><span class="p">,</span> <span class="s2">&quot;8&quot;</span><span class="p">,</span> <span class="s2">&quot;9&quot;</span><span class="p">,</span> <span class="s2">&quot;0&quot;</span><span class="p">,</span> <span class="s2">&quot;1&quot;</span><span class="p">,</span> <span class="s2">&quot;2&quot;</span><span class="p">]:</span>
            <span class="n">names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">name</span> <span class="o">+</span> <span class="n">suffix</span><span class="p">)</span>
    <span class="n">names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;GEO2010&quot;</span><span class="p">)</span>

    <span class="n">c</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="n">nrows</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">columns</span>
    <span class="n">c</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>

    <span class="n">keep</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">c</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
        <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">col</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
                <span class="n">keep</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span>
        <span class="n">filepath</span><span class="p">,</span>
        <span class="n">usecols</span><span class="o">=</span><span class="n">keep</span><span class="p">,</span>
        <span class="n">engine</span><span class="o">=</span><span class="s2">&quot;c&quot;</span><span class="p">,</span>
        <span class="n">na_values</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="mi">99999</span><span class="p">,</span> <span class="o">-</span><span class="mi">999</span><span class="p">],</span>
        <span class="n">converters</span><span class="o">=</span><span class="p">{</span>
            <span class="s2">&quot;GEO2010&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
            <span class="s2">&quot;COUNTY&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
            <span class="s2">&quot;COUSUB&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
            <span class="s2">&quot;DIVISION&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
            <span class="s2">&quot;REGION&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
            <span class="s2">&quot;STATE&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="p">},</span>
    <span class="p">)</span>

    <span class="n">cols</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span>
    <span class="n">fixed</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">cols</span><span class="p">:</span>
        <span class="k">if</span> <span class="n">col</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;D&quot;</span><span class="p">):</span>
            <span class="n">fixed</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;D&quot;</span> <span class="o">+</span> <span class="n">col</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
        <span class="k">elif</span> <span class="n">col</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;N&quot;</span><span class="p">):</span>
            <span class="n">fixed</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;N&quot;</span> <span class="o">+</span> <span class="n">col</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
        <span class="k">elif</span> <span class="n">col</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;1A&quot;</span><span class="p">):</span>
            <span class="n">fixed</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col</span><span class="p">[:</span><span class="o">-</span><span class="mi">2</span><span class="p">]</span> <span class="o">+</span> <span class="s2">&quot;2&quot;</span><span class="p">)</span>

    <span class="n">orig</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">cols</span><span class="p">:</span>
        <span class="k">if</span> <span class="n">col</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;D&quot;</span><span class="p">):</span>
            <span class="n">orig</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">col</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;N&quot;</span><span class="p">):</span>
            <span class="n">orig</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">col</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;1A&quot;</span><span class="p">):</span>
            <span class="n">orig</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>

    <span class="n">renamer</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">orig</span><span class="p">,</span> <span class="n">fixed</span><span class="p">))</span>
    <span class="n">df</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">renamer</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="s2">&quot;columns&quot;</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">[</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">names</span><span class="p">)]]</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">wide_to_long</span><span class="p">(</span>
        <span class="n">df</span><span class="p">,</span> <span class="n">stubnames</span><span class="o">=</span><span class="n">ncdb_vars</span><span class="p">,</span> <span class="n">i</span><span class="o">=</span><span class="s2">&quot;GEO2010&quot;</span><span class="p">,</span> <span class="n">j</span><span class="o">=</span><span class="s2">&quot;year&quot;</span><span class="p">,</span> <span class="n">suffix</span><span class="o">=</span><span class="s2">&quot;(7|8|9|0|1|2)&quot;</span>
    <span class="p">)</span><span class="o">.</span><span class="n">reset_index</span><span class="p">()</span>

    <span class="n">df</span><span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
        <span class="p">{</span><span class="mi">7</span><span class="p">:</span> <span class="mi">1970</span><span class="p">,</span> <span class="mi">8</span><span class="p">:</span> <span class="mi">1980</span><span class="p">,</span> <span class="mi">9</span><span class="p">:</span> <span class="mi">1990</span><span class="p">,</span> <span class="mi">0</span><span class="p">:</span> <span class="mi">2000</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="mi">2010</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="mi">2010</span><span class="p">}</span>
    <span class="p">)</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">groupby</span><span class="p">([</span><span class="s2">&quot;GEO2010&quot;</span><span class="p">,</span> <span class="s2">&quot;year&quot;</span><span class="p">])</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>

    <span class="n">mapper</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()</span><span class="o">.</span><span class="n">ncdb</span><span class="p">,</span> <span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()</span><span class="o">.</span><span class="n">variable</span><span class="p">))</span>

    <span class="n">df</span><span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">mapper</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="s2">&quot;columns&quot;</span><span class="p">)</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">set_index</span><span class="p">(</span><span class="s2">&quot;geoid&quot;</span><span class="p">)</span>

    <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()[</span><span class="s2">&quot;formula&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">dropna</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">df</span><span class="o">.</span><span class="n">eval</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="k">except</span><span class="p">:</span>
            <span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Unable to compute &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">row</span><span class="p">))</span>

    <span class="n">keeps</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">[</span>
        <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">datasets</span><span class="o">.</span><span class="n">codebook</span><span class="p">()[</span><span class="s2">&quot;variable&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;year&quot;</span><span class="p">])</span>
    <span class="p">]</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">keeps</span><span class="p">]</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">df</span><span class="o">.</span><span class="n">n_total_pop</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">]</span>

    <span class="n">df</span><span class="o">.</span><span class="n">to_parquet</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_dir</span><span class="p">,</span> <span class="s2">&quot;ncdb.parquet&quot;</span><span class="p">),</span> <span class="n">compression</span><span class="o">=</span><span class="s2">&quot;brotli&quot;</span><span class="p">)</span>
    <span class="n">storage</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">&quot;ncdb&quot;</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_dir</span><span class="p">,</span> <span class="s2">&quot;ncdb.parquet&quot;</span><span class="p">))</span>
    <span class="n">storage</span><span class="o">.</span><span class="n">build</span><span class="p">(</span><span class="s2">&quot;geosnap_data/storage&quot;</span><span class="p">)</span></div>


<span class="k">def</span> <span class="nf">_fips_filter</span><span class="p">(</span>
    <span class="n">state_fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">county_fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">msa_fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="kc">None</span>
<span class="p">):</span>
    <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

    <span class="n">fips_list</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">each</span> <span class="ow">in</span> <span class="p">[</span><span class="n">state_fips</span><span class="p">,</span> <span class="n">county_fips</span><span class="p">,</span> <span class="n">fips</span><span class="p">]:</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">each</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">,)):</span>
            <span class="n">each</span> <span class="o">=</span> <span class="p">[</span><span class="n">each</span><span class="p">]</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">each</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,)):</span>
            <span class="n">fips_list</span> <span class="o">+=</span> <span class="n">each</span>

    <span class="k">if</span> <span class="n">msa_fips</span><span class="p">:</span>
        <span class="n">fips_list</span> <span class="o">+=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">msa_definitions</span><span class="p">()[</span>
            <span class="n">datasets</span><span class="o">.</span><span class="n">msa_definitions</span><span class="p">()[</span><span class="s2">&quot;CBSA Code&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="n">msa_fips</span>
        <span class="p">][</span><span class="s2">&quot;stcofips&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">geoid</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="nb">tuple</span><span class="p">(</span><span class="n">fips_list</span><span class="p">))]</span>

    <span class="k">return</span> <span class="n">df</span>


<span class="k">def</span> <span class="nf">_from_db</span><span class="p">(</span>
    <span class="n">data</span><span class="p">,</span> <span class="n">state_fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">county_fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">msa_fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fips</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">years</span><span class="o">=</span><span class="kc">None</span>
<span class="p">):</span>

    <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">year</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">years</span><span class="p">)]</span>
    <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">reset_index</span><span class="p">()</span>

    <span class="n">df</span> <span class="o">=</span> <span class="n">_fips_filter</span><span class="p">(</span>
        <span class="n">state_fips</span><span class="o">=</span><span class="n">state_fips</span><span class="p">,</span>
        <span class="n">county_fips</span><span class="o">=</span><span class="n">county_fips</span><span class="p">,</span>
        <span class="n">msa_fips</span><span class="o">=</span><span class="n">msa_fips</span><span class="p">,</span>
        <span class="n">fips</span><span class="o">=</span><span class="n">fips</span><span class="p">,</span>
        <span class="n">data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
    <span class="p">)</span>

    <span class="c1"># we know we&#39;re using 2010, need to drop the year column so no conficts</span>
    <span class="n">tracts</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">tracts_2010</span><span class="p">(</span><span class="n">convert</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
    <span class="n">tracts</span> <span class="o">=</span> <span class="n">tracts</span><span class="p">[[</span><span class="s2">&quot;geoid&quot;</span><span class="p">,</span> <span class="s2">&quot;wkb&quot;</span><span class="p">]]</span>
    <span class="n">tracts</span> <span class="o">=</span> <span class="n">tracts</span><span class="p">[</span><span class="n">tracts</span><span class="o">.</span><span class="n">geoid</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">geoid</span><span class="p">)]</span>
    <span class="n">tracts</span> <span class="o">=</span> <span class="n">convert_gdf</span><span class="p">(</span><span class="n">tracts</span><span class="p">)</span>

    <span class="n">gdf</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">tracts</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">&quot;geoid&quot;</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">&quot;left&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">set_index</span><span class="p">(</span><span class="s2">&quot;geoid&quot;</span><span class="p">)</span>
    <span class="n">gdf</span> <span class="o">=</span> <span class="n">gpd</span><span class="o">.</span><span class="n">GeoDataFrame</span><span class="p">(</span><span class="n">gdf</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">gdf</span>
</pre></div>

    </div>
      
  </div>
</div>
<footer class="footer">
  <div class="container">
    <p class="pull-right">
      <a href="#">Back to top</a>
      
        <br/>
        
      
    </p>
    <p>
        &copy; Copyright 2020, geosnap developers.<br/>
      Created using <a href="http://sphinx-doc.org/">Sphinx</a> 2.4.4.<br/>
    </p>
  </div>
</footer>
  </body>
</html>